import requests
from bs4 import BeautifulSoup
import json
import time
import random
import os
import re
import sys
import io
import unicodedata
import datetime

# 设置控制台输出编码
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')

class MeishichinaRecipeScraper:
    def __init__(self):
        self.base_url = "https://home.meishichina.com/recipe.html"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
        }
        self.recipe_id_counter = 1

        # 创建数据存储文件夹
        self.data_dir = "recipe_data"
        self.json_dir = os.path.join(self.data_dir, "json_files")
        self.log_dir = os.path.join(self.data_dir, "logs")

        # 确保文件夹存在
        for directory in [self.data_dir, self.json_dir, self.log_dir]:
            if not os.path.exists(directory):
                os.makedirs(directory)
                print(f"创建目录: {directory}")

        # 爬取记录文件
        self.log_file = os.path.join(self.log_dir, "crawl_log.json")

        # 加载上次的爬取记录
        self.load_crawl_log()

    def get_page(self, url):
        """Get HTML content from URL with retry mechanism"""
        max_retries = 3
        for attempt in range(max_retries):
            try:
                response = requests.get(url, headers=self.headers, timeout=10)
                response.raise_for_status()
                # 确保使用UTF-8编码
                response.encoding = 'utf-8'
                return response.text
            except requests.exceptions.RequestException as e:
                print(f"获取页面出错 {url}: {e}")
                if attempt < max_retries - 1:
                    sleep_time = 2 * (attempt + 1)
                    print(f"将在 {sleep_time} 秒后重试...")
                    time.sleep(sleep_time)
                else:
                    print("达到最大重试次数，跳过。")
                    return None

    def get_recipe_links(self, page_num=1):
        """Get links to recipe pages from the listing page"""
        if page_num == 1:
            url = self.base_url
        else:
            url = f"https://home.meishichina.com/recipe-list-page-{page_num}.html"

        print(f"正在获取页面: {url}")
        html = self.get_page(url)
        if not html:
            return []

        soup = BeautifulSoup(html, 'html.parser')
        # 尝试选择器，针对最新的网站布局
        recipe_items = soup.select('.big4_list ul li')

        if not recipe_items:
            print("没有找到食谱项目，尝试其他选择器...")
            print(soup.select('title')[0].text if soup.select('title') else "无标题")

            # 尝试其他可能的选择器
            recipe_items = soup.select('.list li')
            if not recipe_items:
                recipe_items = soup.select('.ui-list-item')
            if not recipe_items:
                recipe_items = soup.select('li')

            # 如果上面的都不匹配，查找所有包含recipe的链接
            if not recipe_items or len(recipe_items) == 0:
                all_links = soup.select('a[href*="recipe-"]')
                links = [link['href'] for link in all_links if 'href' in link.attrs and 'recipe-' in link['href']]
                if links:
                    # 过滤出详情页链接，通常格式为 recipe-数字.html
                    links = [link for link in links if re.search(r'recipe-\d+\.html', link)]
                    print(f"通过链接模式找到 {len(links)} 个食谱链接")
                    # 确保是完整URL
                    links = [f"https://home.meishichina.com{link}" if link.startswith('/') else link for link in links]
                    links = [link if link.startswith('http') else f"https://home.meishichina.com/{link}" for link in links]
                    return links

        links = []
        for item in recipe_items:
            # 尝试找到链接元素
            link_elem = item.select_one('a')
            if link_elem and 'href' in link_elem.attrs:
                href = link_elem['href']
                # 确保链接是菜谱详情页
                if 'recipe-' in href and re.search(r'recipe-\d+\.html', href):
                    # 确保是完整URL
                    if not href.startswith('http'):
                        href = f"https://home.meishichina.com{href}" if href.startswith('/') else f"https://home.meishichina.com/{href}"
                    links.append(href)

        print(f"在页面 {page_num} 中找到 {len(links)} 个食谱链接")
        return links

    def parse_recipe(self, url):
        """Parse a single recipe page"""
        print(f"正在解析食谱页面: {url}")
        html = self.get_page(url)
        if not html:
            return None

        soup = BeautifulSoup(html, 'html.parser')

        # 调试信息
        page_title = soup.title.text if soup.title else '无标题'
        print(f"页面标题: {page_title}")

        # Extract recipe name
        name_elem = soup.select_one('h1.recipe_De_title, h1.recipe_title, .recipe_top .title_top_c h1, .recipe-title h1, .recipe_top h1, .recipe-header h1')
        if not name_elem:
            # 如果找不到名称，尝试获取页面中的所有h1标签
            all_h1 = soup.select('h1')
            name_elem = all_h1[0] if all_h1 else None

        name = name_elem.text.strip() if name_elem else "未知菜名"
        # 清理菜名中可能的其他文本
        name = re.sub(r'(独家|\s+)', '', name)
        print(f"食谱名称: {name}")

        # 检查是否为有效的菜谱页面
        if name == "菜谱" or name == "美食天下":
            print("不是有效的菜谱详情页面，跳过")
            return None

        # Extract ingredients - handling different sections
        ingredients = []

        # Main ingredients (主料)
        main_ingredient_sections = soup.select('fieldset.particulars legend')
        found_ingredients = False
        for section in main_ingredient_sections:
            if '主料' in section.text:
                found_ingredients = True
                parent = section.parent
                ingredient_items = parent.select('li')
                for item in ingredient_items:
                    name_span = item.select_one('.category_s1')
                    if name_span:
                        # Find ingredient name from a elements inside span
                        a_elems = name_span.select('a')
                        if a_elems:
                            for a in a_elems:
                                ing_name = a.text.strip()
                                if ing_name:
                                    ingredients.append(ing_name)
                        else:
                            # Extract ingredient name directly from span
                            ing_name = name_span.text.strip()
                            # Clean up the name, removing any nested tag text
                            ing_name = re.sub(r'[0-9.]+$', '', ing_name).strip()
                            ingredients.append(ing_name)

        # Auxiliary ingredients (辅料)
        for section in main_ingredient_sections:
            if '辅料' in section.text:
                found_ingredients = True
                parent = section.parent
                ingredient_items = parent.select('li')
                for item in ingredient_items:
                    name_span = item.select_one('.category_s1')
                    if name_span:
                        a_elems = name_span.select('a')
                        if a_elems:
                            for a in a_elems:
                                ing_name = a.text.strip()
                                if ing_name:
                                    ingredients.append(ing_name)
                        else:
                            ing_name = name_span.text.strip()
                            ing_name = re.sub(r'[0-9.]+$', '', ing_name).strip()
                            ingredients.append(ing_name)

        # Seasonings (调料)
        for section in main_ingredient_sections:
            if '调料' in section.text:
                found_ingredients = True
                parent = section.parent
                ingredient_items = parent.select('li')
                for item in ingredient_items:
                    name_span = item.select_one('.category_s1')
                    if name_span:
                        a_elems = name_span.select('a')
                        if a_elems:
                            for a in a_elems:
                                ing_name = a.text.strip()
                                if ing_name:
                                    ingredients.append(ing_name)
                        else:
                            ing_name = name_span.text.strip()
                            ing_name = re.sub(r'[0-9.]+$', '', ing_name).strip()
                            ingredients.append(ing_name)

        # 尝试从food-material中提取
        if not found_ingredients:
            food_material = soup.select_one('.food-material')
            if food_material:
                for item in food_material.select('li'):
                    text = item.text.strip()
                    parts = text.split()
                    if parts:
                        ingredients.append(parts[0])

        # If we couldn't find ingredients with the new selectors, try the old ones
        if not ingredients:
            ingredient_selectors = [
                '.recipe_material_show .recipeCategory_sub_block ul li',
                '.recipeCategory_sub_block ul li',
                '.recipe-ingrediants li',
                '.recipe-ingredients li',
                '.recipe_ingredients li'
            ]

            for selector in ingredient_selectors:
                ingredients_section = soup.select(selector)
                if ingredients_section:
                    for ing in ingredients_section:
                        ing_text = ing.text.strip()
                        if ing_text:
                            # Extract only the ingredient name, removing measurement
                            match = re.match(r'^([^0-9]+)', ing_text)
                            if match:
                                clean_ing = match.group(1).strip()
                                ingredients.append(clean_ing)
                    break  # 找到匹配的选择器后就停止

        # 移除HTML导航元素和重复的元素
        clean_ingredients = []
        blacklist = ['美食天下', '首页', '菜谱', '专题', '笔记', '社区', '活动', '搜索', '饮食健康', '烘焙', '妈妈派', '客户端']
        for ing in ingredients:
            if ing not in blacklist and ing not in clean_ingredients:
                clean_ingredients.append(ing)

        ingredients = clean_ingredients

        if not ingredients:
            print("未找到食材信息")
        else:
            print(f"找到 {len(ingredients)} 个食材")

        # Extract steps
        steps = []

        # Try with recipeStep selector
        step_items = soup.select('.recipeStep li')
        for item in step_items:
            step_word = item.select_one('.recipeStep_word')
            if step_word:
                # Remove the step number
                for grey_div in step_word.select('.grey'):
                    grey_div.decompose()
                step_text = step_word.text.strip()
                if step_text:
                    steps.append(step_text)

        # If we couldn't find steps with the new selector, try the old ones
        if not steps:
            step_selectors = [
                '.recipe_step .recipeStep_item .recipe_step_word',
                '.recipeStep_item .recipe_step_word',
                '.recipe-steps li',
                '.recipe_steps li',
                '.recipeStep_word'
            ]

            for selector in step_selectors:
                steps_section = soup.select(selector)
                if steps_section:
                    for step in steps_section:
                        # Remove step number if present
                        for grey_div in step.select('.grey'):
                            grey_div.decompose()
                        step_text = step.text.strip()
                        if step_text:
                            steps.append(step_text)
                    break  # 找到匹配的选择器后就停止

        # 清理步骤中的广告和版权信息
        clean_steps = []
        for step in steps:
            if not any(skip in step for skip in ['禁止其他平台或个人转载', '本菜谱为作者独家', '版权所有']):
                clean_steps.append(step)

        steps = clean_steps

        if not steps:
            print("未找到烹饪步骤")
        else:
            print(f"找到 {len(steps)} 个烹饪步骤")

        # Extract tags
        tags = []

        # Try with recipe category selectors
        tag_items = soup.select('.recipeCategory_sub_R li .category_s1 a')
        for tag_item in tag_items:
            tag_text = tag_item.text.strip()
            if tag_text and tag_text not in tags:
                tags.append(tag_text)

        # Also look for tags in path
        path_tags = soup.select('#path a.vest')
        for tag_item in path_tags:
            tag_text = tag_item.text.strip()
            if tag_text and tag_text not in tags:
                tags.append(tag_text)

        # If we couldn't find tags with the new selectors, try the old ones
        if not tags:
            tag_selectors = [
                '.recipe_info_tag a',
                '.recipe-tags a',
                '.recipe_tags a',
            ]

            for selector in tag_selectors:
                tags_section = soup.select(selector)
                if tags_section:
                    for tag in tags_section:
                        tag_text = tag.text.strip()
                        if tag_text and tag_text != '添加标签' and tag_text not in tags:
                            tags.append(tag_text)
                    break  # 找到匹配的选择器后就停止

        # 清理掉已经在食材列表中的标签
        tags = [tag for tag in tags if tag not in ingredients]

        # 非素食关键词列表
        non_veg_keywords = [
            '肉', '鸡', '鸭', '鱼', '牛', '羊', '猪', '虾', '蟹', '贝', '海鲜',
            '培根', '火腿', '香肠', '肝', '胗', '肚', '肠', '肉汤', '高汤', '鸡汤',
            '鱼汤', '骨汤', '猪油', '鸡蛋', '海参', '鹅', '兔', '鹿'
        ]

        # 定义确定为非素食的完整词汇
        def is_non_vegetarian(text):
            # 需要精确匹配的非素食词汇
            exact_match_terms = {
                '猪油', '牛油', '鱼油', '火腿', '香肠', '培根', '肉松', '鱼丸', '虾饺'
            }

            # 先检查是否包含精确匹配的非素食词汇
            for term in exact_match_terms:
                if term in text:
                    return True

            # 预先定义的非素食产品
            complete_non_veg_items = {
                '瘦肉', '五花肉', '里脊', '牛肉', '羊肉', '鸡肉', '鸭肉', '鱼肉',
                '猪肉', '肉末', '肉丝', '肉片', '培根', '火腿', '香肠', '午餐肉',
                '排骨', '鸡翅', '鸡腿', '鸡胸', '牛腩', '牛排', '大虾', '鱼排',
                '猪排', '鸡排', '肉馅', '虾仁', '鱼片', '鱼丸', '牛筋', '鸡爪',
                '海参', '干贝', '鲍鱼', '墨鱼', '章鱼', '鱿鱼', '海蜇', '海带'
            }

            # 检查是否包含完整非素食词汇
            for item in complete_non_veg_items:
                if item in text:
                    return True

            # 单字判断的特殊关键词（任何包含这些字的都是非素食）
            single_char_keywords = {'鹅', '兔', '鹿'}
            for keyword in single_char_keywords:
                if keyword in text:
                    return True

            # 检查关键词 - 排除特定的素食误判词汇
            for keyword in non_veg_keywords:
                if keyword in text:
                    # 排除误判情况
                    false_positives = {
                        '牛': ['牛奶', '牛油果', '花牛', '牛蒡'],
                        '鱼': ['鱼腥草', '鱼香', '鱼豆腐'],
                        '鸡': ['鸡蛋糕', '田鸡'],
                        '蛋': ['皮蛋豆腐', '松花蛋'],
                    }

                    # 如果当前关键词有可能的误判情况
                    if keyword in false_positives:
                        # 检查是否是误判情况
                        is_false_positive = any(fp in text for fp in false_positives[keyword])
                        # 如果不是误判，则确实是非素食
                        if not is_false_positive:
                            return True
                    else:
                        # 如果该关键词没有列在误判列表中，则确定是非素食
                        return True

            return False

        # 判断是否素食
        is_vegetarian = True

        # 检查菜名
        if is_non_vegetarian(name.lower()):
            is_vegetarian = False

        # 检查食材
        if is_vegetarian:
            for ing in ingredients:
                if is_non_vegetarian(ing.lower()):
                    is_vegetarian = False
                    break

        # Extract image
        image = ""
        # Try recipe_De_imgBox first
        img_elem = soup.select_one('.recipe_De_imgBox img')
        if img_elem and 'src' in img_elem.attrs:
            image = img_elem['src']
        else:
            # Fallback to other selectors
            img_selectors = [
                '.recipe_topimg_big img',
                '.recipe-top-img img',
                '.recipe_pic img',
                '.recipe-pic img'
            ]

            for selector in img_selectors:
                img_elem = soup.select_one(selector)
                if img_elem and 'src' in img_elem.attrs:
                    image = img_elem['src']
                    break  # 找到匹配的选择器后就停止

        # 检查数据完整性
        if not ingredients or not steps:
            print("数据不完整，尝试进一步解析...")
            # 尝试获取所有段落文本，可能包含食谱信息
            paragraphs = soup.select('p')
            for p in paragraphs[:10]:  # 只查看前10个段落
                if len(p.text.strip()) > 20:  # 只考虑内容足够的段落
                    steps.append(p.text.strip())

            # 尝试获取所有列表项
            list_items = soup.select('li')
            for li in list_items[:20]:  # 只查看前20个列表项
                text = li.text.strip()
                if len(text) > 3 and len(text) < 30 and text not in blacklist:  # 长度适中的可能是食材
                    ingredients.append(text)

        # 如果仍然无法获取足够的数据，可能不是有效的菜谱页面
        if not ingredients and not steps:
            print("无法获取有效的菜谱数据，可能不是菜谱页面")
            return None

        recipe = {
            "id": self.recipe_id_counter,
            "name": name,
            "ingredients": ingredients,
            "steps": steps,
            "tags": tags,
            "isVegetarian": is_vegetarian,
            "image": image
        }

        self.recipe_id_counter += 1
        return recipe

    def load_crawl_log(self):
        """加载上次的爬取记录"""
        if os.path.exists(self.log_file):
            try:
                with open(self.log_file, 'r', encoding='utf-8') as f:
                    self.crawl_log = json.load(f)
                print(f"已加载爬取记录: 上次爬取到第 {self.crawl_log.get('last_page', 1)} 页")

                # 更新recipe_id_counter
                if 'last_recipe_id' in self.crawl_log:
                    self.recipe_id_counter = self.crawl_log['last_recipe_id'] + 1
            except Exception as e:
                print(f"加载爬取记录出错: {e}")
                self.crawl_log = {"crawl_history": []}
        else:
            self.crawl_log = {"crawl_history": []}

    def save_crawl_log(self, current_page, recipes_count, output_filename=None):
        """保存当前爬取记录"""
        timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")

        # 更新最后一次爬取信息
        self.crawl_log["last_page"] = current_page
        self.crawl_log["last_recipe_id"] = self.recipe_id_counter - 1
        self.crawl_log["last_crawl_time"] = timestamp

        # 记录本次输出的JSON文件
        if output_filename:
            self.crawl_log["last_output_file"] = output_filename

        # 添加本次爬取记录
        crawl_record = {
            "crawl_time": timestamp,
            "start_page": self.crawl_log.get("last_page", 1),
            "end_page": current_page,
            "recipes_count": recipes_count,
            "total_recipes": self.recipe_id_counter - 1
        }

        # 添加输出文件名到记录
        if output_filename:
            crawl_record["output_file"] = output_filename

        self.crawl_log["crawl_history"].append(crawl_record)

        # 保存记录
        with open(self.log_file, 'w', encoding='utf-8') as f:
            json.dump(self.crawl_log, f, ensure_ascii=False, indent=2)

        print(f"已保存爬取记录: 当前爬取到第 {current_page} 页，共 {recipes_count} 个食谱")
        if output_filename:
            print(f"输出文件: {output_filename}")

    def scrape_recipes(self, count=100, start_page=None):
        """Scrape specified number of recipes starting from given page"""
        recipes = []

        # 如果没有指定起始页，使用记录中的最后一页
        if start_page is None:
            start_page = self.crawl_log.get("last_page", 1)
            print(f"从上次爬取的页面继续: 第 {start_page} 页")

        current_page = start_page
        attempts = 0
        max_attempts = count * 2  # 最多尝试抓取的次数，避免无限循环

        # 保存本次爬取的临时文件列表
        temp_files = []

        while len(recipes) < count and attempts < max_attempts:
            print(f"正在获取第 {current_page} 页的食谱...")
            links = self.get_recipe_links(current_page)

            if not links:
                print(f"在第 {current_page} 页未找到食谱链接。尝试下一页...")
                current_page += 1
                if current_page > start_page + 5:  # 如果连续5页都没有找到链接，则退出
                    print("连续多页未找到链接，退出抓取。")
                    break
                continue

            for link in links:
                if len(recipes) >= count:
                    break

                attempts += 1
                print(f"正在抓取第 {self.recipe_id_counter} 个食谱，链接: {link}")
                recipe = self.parse_recipe(link)

                if recipe:
                    # 确保菜谱数据有基本信息才添加
                    if recipe['name'] != "未知菜名" and (recipe['ingredients'] or recipe['steps']):
                        recipes.append(recipe)
                        print(f"成功添加食谱: {recipe['name']}")
                    else:
                        print(f"食谱 {recipe['name']} 信息不完整，跳过")

                # Add delay to avoid overloading the server
                delay = random.uniform(1, 3)
                print(f"等待 {delay:.2f} 秒...")
                time.sleep(delay)

            current_page += 1

            # 保存中间结果，防止中途中断丢失数据
            if recipes and len(recipes) % 10 == 0:
                temp_filename = os.path.join(self.json_dir, f"recipes_temp_{len(recipes)}.json")
                self.save_to_json(recipes, temp_filename)
                temp_files.append(temp_filename)
                print(f"已保存 {len(recipes)} 个食谱到临时文件")

        # 获取最终输出文件名
        final_output = None
        if recipes:
            # 使用时间戳生成最终文件名
            timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
            final_output = os.path.join(self.json_dir, f"recipes_{timestamp}.json")
            self.save_to_json(recipes, final_output)

        # 保存爬取记录，包含输出文件名
        self.save_crawl_log(current_page - 1, len(recipes), final_output)

        return recipes, final_output

    def save_to_json(self, recipes, filename=None):
        """Save recipes to JSON file"""
        if filename is None:
            # 使用时间戳生成唯一文件名
            timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
            filename = os.path.join(self.json_dir, f"recipes_{timestamp}.json")

        with open(filename, 'w', encoding='utf-8') as f:
            json.dump(recipes, f, ensure_ascii=False, indent=2)
        print(f"已保存 {len(recipes)} 个食谱到文件 {filename}")
        return filename


if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser(description='从美食天下网站爬取菜谱')
    parser.add_argument('--count', type=int, default=100, help='要爬取的菜谱数量 (默认: 100)')
    parser.add_argument('--start-page', type=int, help='起始页码 (默认: 使用上次爬取的页码)')
    parser.add_argument('--output', type=str, help='输出JSON文件名 (默认: 自动生成)')
    parser.add_argument('--debug', action='store_true', help='启用调试模式')

    args = parser.parse_args()

    if args.debug:
        print("调试模式已启用")
        # 测试单个页面的链接提取
        scraper = MeishichinaRecipeScraper()
        links = scraper.get_recipe_links(args.start_page or 1)
        print(f"找到 {len(links)} 个链接:")
        for i, link in enumerate(links[:5]):  # 只显示前5个链接
            print(f"{i+1}. {link}")

        if links:
            # 测试单个食谱的解析
            print("\n测试解析第一个食谱:")
            recipe = scraper.parse_recipe(links[0])
            if recipe:
                print(json.dumps(recipe, ensure_ascii=False, indent=2))
    else:
        print(f"开始爬取 {args.count} 条菜谱...")
        scraper = MeishichinaRecipeScraper()
        recipes, output_file = scraper.scrape_recipes(count=args.count, start_page=args.start_page)

        if args.output:
            # 如果指定了输出文件名
            output_path = os.path.join(scraper.json_dir, args.output)
            output_file = scraper.save_to_json(recipes, filename=output_path)
            # 更新爬取记录中的输出文件
            scraper.save_crawl_log(
                scraper.crawl_log["last_page"],
                len(recipes),
                output_file
            )
